{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Algorithm"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In this tutorial, we provide an example of creaing a new SSL algorithm by resusing the component hook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from semilearn import get_dataset, get_data_loader, get_net_builder, get_algorithm, get_config, Trainer\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step1: Create and Register Algorithm"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will create a new algorithm based on FixMatch and combine it with entropy loss, using the following steps:\n",
    "\n",
    "* Inherit FixMatch algorithm\n",
    "* Rewrite the 'train_step' function by adding entropy loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from semilearn.core.utils import ALGORITHMS\n",
    "from semilearn.algorithms.fixmatch import FixMatch\n",
    "from inspect import signature\n",
    "\n",
    "\n",
    "\n",
    "def entropy_loss(ul_y):\n",
    "    p = F.softmax(ul_y, dim=1)\n",
    "    return -(p * F.log_softmax(ul_y, dim=1)).sum(dim=1).mean(dim=0)\n",
    "\n",
    "\n",
    "@ALGORITHMS.register('fixmatch_entropy')\n",
    "class FixMatchEntropy(FixMatch):\n",
    "    def train_step(self, idx_lb, x_lb, y_lb, x_ulb_w, x_ulb_s):\n",
    "        num_lb = y_lb.shape[0]\n",
    "\n",
    "        # inference and calculate sup/unsup losses\n",
    "        with self.amp_cm():\n",
    "            if self.use_cat:\n",
    "                inputs = torch.cat((x_lb, x_ulb_w, x_ulb_s))\n",
    "                outputs = self.model(inputs)\n",
    "                logits_x_lb = outputs['logits'][:num_lb]\n",
    "                logits_x_ulb_w, logits_x_ulb_s = outputs['logits'][num_lb:].chunk(2)\n",
    "                feats_x_lb = outputs['feat'][:num_lb]\n",
    "                feats_x_ulb_w, feats_x_ulb_s = outputs['feat'][num_lb:].chunk(2)\n",
    "            else:\n",
    "                outs_x_lb = self.model(x_lb) \n",
    "                logits_x_lb = outs_x_lb['logits']\n",
    "                feats_x_lb = outs_x_lb['feat']\n",
    "                outs_x_ulb_s = self.model(x_ulb_s)\n",
    "                logits_x_ulb_s = outs_x_ulb_s['logits']\n",
    "                feats_x_ulb_s = outs_x_ulb_s['feat']\n",
    "                with torch.no_grad():\n",
    "                    outs_x_ulb_w = self.model(x_ulb_w)\n",
    "                    logits_x_ulb_w = outs_x_ulb_w['logits']\n",
    "                    feats_x_ulb_w = outs_x_ulb_w['feat']\n",
    "            feat_dict = {'x_lb':feats_x_lb, 'x_ulb_w':feats_x_ulb_w, 'x_ulb_s':feats_x_ulb_s}\n",
    "\n",
    "            sup_loss = self.ce_loss(logits_x_lb, y_lb, reduction='mean')\n",
    "            \n",
    "            # probs_x_ulb_w = torch.softmax(logits_x_ulb_w, dim=-1)\n",
    "            probs_x_ulb_w = self.compute_prob(logits_x_ulb_w.detach())\n",
    "            \n",
    "            # if distribution alignment hook is registered, call it \n",
    "            # this is implemented for imbalanced algorithm - CReST\n",
    "            if self.registered_hook(\"DistAlignHook\"):\n",
    "                probs_x_ulb_w = self.call_hook(\"dist_align\", \"DistAlignHook\", probs_x_ulb=probs_x_ulb_w.detach())\n",
    "\n",
    "            # compute mask\n",
    "            mask = self.call_hook(\"masking\", \"MaskingHook\", logits_x_ulb=probs_x_ulb_w, softmax_x_ulb=False)\n",
    "\n",
    "            # generate unlabeled targets using pseudo label hook\n",
    "            pseudo_label = self.call_hook(\"gen_ulb_targets\", \"PseudoLabelingHook\", \n",
    "                                          logits=probs_x_ulb_w,\n",
    "                                          use_hard_label=self.use_hard_label,\n",
    "                                          T=self.T,\n",
    "                                          softmax=False)\n",
    "\n",
    "            unsup_loss = self.consistency_loss(logits_x_ulb_s,\n",
    "                                               pseudo_label,\n",
    "                                               'ce',\n",
    "                                               mask=mask)\n",
    "            \n",
    "            # NOTE: add entropy loss here\n",
    "            loss_entmin = entropy_loss(logits_x_ulb_w)\n",
    "\n",
    "            total_loss = sup_loss + self.lambda_u * unsup_loss\n",
    "\n",
    "        out_dict = self.process_out_dict(loss=total_loss, feat=feat_dict)\n",
    "        log_dict = self.process_log_dict(sup_loss=sup_loss.item(), \n",
    "                                         unsup_loss=unsup_loss.item(), \n",
    "                                         total_loss=total_loss.item(), \n",
    "                                         util_ratio=mask.float().mean().item())\n",
    "        return out_dict, log_dict"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: define configs and create config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/bin/sh: 1: netstat: not found\n"
     ]
    }
   ],
   "source": [
    "config = {\n",
    "    'algorithm': 'fixmatch_entropy',\n",
    "    'net': 'vit_tiny_patch2_32',\n",
    "    'use_pretrain': True, \n",
    "    'pretrain_path': 'https://github.com/microsoft/Semi-supervised-learning/releases/download/v.0.0.0/vit_tiny_patch2_32_mlp_im_1k_32.pth',\n",
    "\n",
    "    # optimization configs\n",
    "    'epoch': 1,  # set to 100\n",
    "    'num_train_iter': 5000,  # set to 102400\n",
    "    'num_eval_iter': 500,   # set to 1024\n",
    "    'num_log_iter': 50,    # set to 256\n",
    "    'optim': 'AdamW',\n",
    "    'lr': 5e-4,\n",
    "    'layer_decay': 0.5,\n",
    "    'batch_size': 16,\n",
    "    'eval_batch_size': 16,\n",
    "\n",
    "\n",
    "    # dataset configs\n",
    "    'dataset': 'cifar10',\n",
    "    'num_labels': 40,\n",
    "    'num_classes': 10,\n",
    "    'img_size': 32,\n",
    "    'crop_ratio': 0.875,\n",
    "    'data_dir': './data',\n",
    "    'ulb_samples_per_class': None,\n",
    "\n",
    "    # algorithm specific configs\n",
    "    'hard_label': True,\n",
    "    'uratio': 2,\n",
    "    'ulb_loss_ratio': 1.0,\n",
    "\n",
    "    # device configs\n",
    "    'gpu': 0,\n",
    "    'world_size': 1,\n",
    "    'distributed': False,\n",
    "    \"num_workers\": 2,\n",
    "}\n",
    "config = get_config(config)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: create algorithm and dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "lb count: [4, 4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "ulb count: [5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000]\n",
      "Files already downloaded and verified\n",
      "unlabeled data number: 50000, labeled data number 40\n",
      "Create train and test data loaders\n",
      "[!] data loader keys: dict_keys(['train_lb', 'train_ulb', 'eval'])\n",
      "_IncompatibleKeys(missing_keys=['head.weight', 'head.bias'], unexpected_keys=[])\n",
      "Create optimizer and scheduler\n"
     ]
    }
   ],
   "source": [
    "algorithm = get_algorithm(config,  get_net_builder(config.net, from_name=False), tb_log=None, logger=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "lb count: [4, 4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "ulb count: [5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000]\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "dataset_dict = get_dataset(config, config.algorithm, config.dataset, config.num_labels, config.num_classes, data_dir=config.data_dir, include_lb_to_ulb=config.include_lb_to_ulb)\n",
    "train_lb_loader = get_data_loader(config, dataset_dict['train_lb'], config.batch_size)\n",
    "train_ulb_loader = get_data_loader(config, dataset_dict['train_ulb'], int(config.batch_size * config.uratio))\n",
    "eval_loader = get_data_loader(config, dataset_dict['eval'], config.eval_batch_size)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: train and evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "50 iteration USE_EMA: True, train/sup_loss: 3.5051, train/unsup_loss: 0.0000, train/total_loss: 3.5051, train/util_ratio: 0.0000, train/run_time: 0.1370, lr: 0.0005, train/prefecth_time: 0.0036 \n",
      "100 iteration USE_EMA: True, train/sup_loss: 1.0864, train/unsup_loss: 0.0000, train/total_loss: 1.0864, train/util_ratio: 0.0000, train/run_time: 0.1388, lr: 0.0005, train/prefecth_time: 0.0031 \n",
      "150 iteration USE_EMA: True, train/sup_loss: 0.5159, train/unsup_loss: 0.0000, train/total_loss: 0.5159, train/util_ratio: 0.0000, train/run_time: 0.1384, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "200 iteration USE_EMA: True, train/sup_loss: 0.5048, train/unsup_loss: 0.0000, train/total_loss: 0.5048, train/util_ratio: 0.0000, train/run_time: 0.1384, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "250 iteration USE_EMA: True, train/sup_loss: 0.5692, train/unsup_loss: 0.1162, train/total_loss: 0.6854, train/util_ratio: 0.1562, train/run_time: 0.1395, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "300 iteration USE_EMA: True, train/sup_loss: 0.1154, train/unsup_loss: 0.0059, train/total_loss: 0.1214, train/util_ratio: 0.0625, train/run_time: 0.1391, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "350 iteration USE_EMA: True, train/sup_loss: 0.0512, train/unsup_loss: 0.0717, train/total_loss: 0.1230, train/util_ratio: 0.0938, train/run_time: 0.1396, lr: 0.0005, train/prefecth_time: 0.0036 \n",
      "400 iteration USE_EMA: True, train/sup_loss: 0.1361, train/unsup_loss: 0.0063, train/total_loss: 0.1425, train/util_ratio: 0.1250, train/run_time: 0.1401, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "450 iteration USE_EMA: True, train/sup_loss: 0.0383, train/unsup_loss: 0.0889, train/total_loss: 0.1272, train/util_ratio: 0.3125, train/run_time: 0.1403, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "validating...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix:\n",
      "[[0.    0.    0.    0.    0.    0.    0.    1.    0.    0.   ]\n",
      " [0.    0.    0.    0.    0.    0.    0.    1.    0.    0.   ]\n",
      " [0.    0.    0.    0.    0.    0.    0.    1.    0.    0.   ]\n",
      " [0.    0.    0.    0.004 0.    0.001 0.    0.995 0.    0.   ]\n",
      " [0.    0.    0.    0.002 0.    0.    0.    0.998 0.    0.   ]\n",
      " [0.    0.    0.    0.001 0.    0.014 0.    0.985 0.    0.   ]\n",
      " [0.    0.    0.    0.    0.    0.    0.    1.    0.    0.   ]\n",
      " [0.    0.    0.    0.    0.    0.    0.    1.    0.    0.   ]\n",
      " [0.    0.    0.    0.    0.    0.    0.    1.    0.    0.   ]\n",
      " [0.    0.    0.    0.    0.    0.    0.    1.    0.    0.   ]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "500 iteration, USE_EMA: True, train/sup_loss: 0.0129, train/unsup_loss: 0.0249, train/total_loss: 0.0378, train/util_ratio: 0.1250, train/run_time: 0.1401, eval/loss: 5.5155, eval/top-1-acc: 0.1018, eval/balanced_acc: 0.1018, eval/precision: 0.1605, eval/recall: 0.1018, eval/F1: 0.0218, lr: 0.0005, train/prefecth_time: 0.0035 BEST_EVAL_ACC: 0.1018, at 500 iters\n",
      "550 iteration USE_EMA: True, train/sup_loss: 0.0389, train/unsup_loss: 0.1663, train/total_loss: 0.2052, train/util_ratio: 0.3125, train/run_time: 0.1406, lr: 0.0005, train/prefecth_time: 0.0039 \n",
      "600 iteration USE_EMA: True, train/sup_loss: 0.0373, train/unsup_loss: 0.1409, train/total_loss: 0.1782, train/util_ratio: 0.3438, train/run_time: 0.1402, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "650 iteration USE_EMA: True, train/sup_loss: 0.0120, train/unsup_loss: 0.4798, train/total_loss: 0.4917, train/util_ratio: 0.4062, train/run_time: 0.1406, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "700 iteration USE_EMA: True, train/sup_loss: 0.0063, train/unsup_loss: 0.1671, train/total_loss: 0.1734, train/util_ratio: 0.3438, train/run_time: 0.1410, lr: 0.0005, train/prefecth_time: 0.0036 \n",
      "750 iteration USE_EMA: True, train/sup_loss: 0.1073, train/unsup_loss: 0.4499, train/total_loss: 0.5573, train/util_ratio: 0.4688, train/run_time: 0.1413, lr: 0.0005, train/prefecth_time: 0.0040 \n",
      "800 iteration USE_EMA: True, train/sup_loss: 0.0048, train/unsup_loss: 0.3912, train/total_loss: 0.3960, train/util_ratio: 0.5938, train/run_time: 0.1408, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "850 iteration USE_EMA: True, train/sup_loss: 0.0266, train/unsup_loss: 0.3070, train/total_loss: 0.3336, train/util_ratio: 0.5000, train/run_time: 0.1407, lr: 0.0005, train/prefecth_time: 0.0039 \n",
      "900 iteration USE_EMA: True, train/sup_loss: 0.0046, train/unsup_loss: 0.0333, train/total_loss: 0.0378, train/util_ratio: 0.4375, train/run_time: 0.1407, lr: 0.0005, train/prefecth_time: 0.0039 \n",
      "950 iteration USE_EMA: True, train/sup_loss: 0.0058, train/unsup_loss: 0.0415, train/total_loss: 0.0473, train/util_ratio: 0.4375, train/run_time: 0.1401, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "validating...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix:\n",
      "[[0.303 0.    0.006 0.004 0.    0.003 0.    0.645 0.    0.039]\n",
      " [0.    0.001 0.    0.    0.    0.001 0.    0.292 0.    0.706]\n",
      " [0.    0.    0.506 0.018 0.    0.11  0.    0.361 0.    0.005]\n",
      " [0.    0.    0.001 0.495 0.    0.263 0.    0.239 0.    0.002]\n",
      " [0.001 0.    0.005 0.061 0.    0.123 0.    0.809 0.    0.001]\n",
      " [0.    0.    0.    0.031 0.    0.78  0.    0.189 0.    0.   ]\n",
      " [0.001 0.    0.028 0.064 0.    0.231 0.002 0.67  0.    0.004]\n",
      " [0.    0.    0.    0.004 0.    0.127 0.    0.868 0.    0.001]\n",
      " [0.036 0.    0.004 0.004 0.    0.031 0.    0.677 0.    0.248]\n",
      " [0.    0.    0.    0.002 0.    0.001 0.    0.121 0.    0.876]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "1000 iteration, USE_EMA: True, train/sup_loss: 0.0189, train/unsup_loss: 0.2167, train/total_loss: 0.2356, train/util_ratio: 0.5000, train/run_time: 0.1406, eval/loss: 2.4450, eval/top-1-acc: 0.3831, eval/balanced_acc: 0.3831, eval/precision: 0.5644, eval/recall: 0.3831, eval/F1: 0.3187, lr: 0.0005, train/prefecth_time: 0.0036 BEST_EVAL_ACC: 0.3831, at 1000 iters\n",
      "1050 iteration USE_EMA: True, train/sup_loss: 0.0346, train/unsup_loss: 0.0441, train/total_loss: 0.0788, train/util_ratio: 0.6875, train/run_time: 0.1408, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "1100 iteration USE_EMA: True, train/sup_loss: 0.0026, train/unsup_loss: 0.3319, train/total_loss: 0.3345, train/util_ratio: 0.5938, train/run_time: 0.1411, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "1150 iteration USE_EMA: True, train/sup_loss: 0.1744, train/unsup_loss: 0.3718, train/total_loss: 0.5462, train/util_ratio: 0.5312, train/run_time: 0.1416, lr: 0.0005, train/prefecth_time: 0.0040 \n",
      "1200 iteration USE_EMA: True, train/sup_loss: 0.0111, train/unsup_loss: 0.1394, train/total_loss: 0.1504, train/util_ratio: 0.6250, train/run_time: 0.1416, lr: 0.0005, train/prefecth_time: 0.0040 \n",
      "1250 iteration USE_EMA: True, train/sup_loss: 0.0191, train/unsup_loss: 0.2628, train/total_loss: 0.2818, train/util_ratio: 0.6250, train/run_time: 0.1412, lr: 0.0005, train/prefecth_time: 0.0036 \n",
      "1300 iteration USE_EMA: True, train/sup_loss: 0.0078, train/unsup_loss: 0.1025, train/total_loss: 0.1103, train/util_ratio: 0.5625, train/run_time: 0.1416, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "1350 iteration USE_EMA: True, train/sup_loss: 0.0248, train/unsup_loss: 0.6699, train/total_loss: 0.6947, train/util_ratio: 0.6562, train/run_time: 0.1408, lr: 0.0005, train/prefecth_time: 0.0039 \n",
      "1400 iteration USE_EMA: True, train/sup_loss: 0.0063, train/unsup_loss: 0.3722, train/total_loss: 0.3785, train/util_ratio: 0.7188, train/run_time: 0.1411, lr: 0.0005, train/prefecth_time: 0.0034 \n",
      "1450 iteration USE_EMA: True, train/sup_loss: 0.0029, train/unsup_loss: 0.0657, train/total_loss: 0.0686, train/util_ratio: 0.6875, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.956 0.    0.003 0.002 0.    0.002 0.002 0.011 0.001 0.023]\n",
      " [0.    0.88  0.    0.    0.    0.    0.    0.    0.    0.12 ]\n",
      " [0.046 0.    0.669 0.017 0.003 0.117 0.005 0.137 0.    0.006]\n",
      " [0.002 0.001 0.009 0.606 0.002 0.288 0.011 0.077 0.001 0.003]\n",
      " [0.018 0.    0.012 0.039 0.387 0.081 0.003 0.46  0.    0.   ]\n",
      " [0.    0.    0.    0.044 0.    0.868 0.    0.088 0.    0.   ]\n",
      " [0.007 0.    0.023 0.005 0.    0.026 0.918 0.019 0.    0.002]\n",
      " [0.001 0.    0.001 0.004 0.006 0.15  0.    0.836 0.    0.002]\n",
      " [0.293 0.004 0.005 0.006 0.    0.012 0.    0.022 0.279 0.379]\n",
      " [0.009 0.003 0.    0.001 0.    0.    0.    0.    0.    0.987]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "1500 iteration, USE_EMA: True, train/sup_loss: 0.0016, train/unsup_loss: 0.1244, train/total_loss: 0.1260, train/util_ratio: 0.5938, train/run_time: 0.1409, eval/loss: 0.7701, eval/top-1-acc: 0.7386, eval/balanced_acc: 0.7386, eval/precision: 0.8133, eval/recall: 0.7386, eval/F1: 0.7264, lr: 0.0005, train/prefecth_time: 0.0034 BEST_EVAL_ACC: 0.7386, at 1500 iters\n",
      "1550 iteration USE_EMA: True, train/sup_loss: 0.0057, train/unsup_loss: 0.2125, train/total_loss: 0.2182, train/util_ratio: 0.5938, train/run_time: 0.1414, lr: 0.0005, train/prefecth_time: 0.0144 \n",
      "1600 iteration USE_EMA: True, train/sup_loss: 0.0051, train/unsup_loss: 0.4610, train/total_loss: 0.4661, train/util_ratio: 0.6875, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0031 \n",
      "1650 iteration USE_EMA: True, train/sup_loss: 0.0011, train/unsup_loss: 0.2337, train/total_loss: 0.2348, train/util_ratio: 0.8438, train/run_time: 0.1416, lr: 0.0004, train/prefecth_time: 0.0029 \n",
      "1700 iteration USE_EMA: True, train/sup_loss: 0.0014, train/unsup_loss: 0.1703, train/total_loss: 0.1717, train/util_ratio: 0.6562, train/run_time: 0.1406, lr: 0.0004, train/prefecth_time: 0.0035 \n",
      "1750 iteration USE_EMA: True, train/sup_loss: 0.2324, train/unsup_loss: 0.1236, train/total_loss: 0.3560, train/util_ratio: 0.5938, train/run_time: 0.1411, lr: 0.0004, train/prefecth_time: 0.0034 \n",
      "1800 iteration USE_EMA: True, train/sup_loss: 0.0040, train/unsup_loss: 0.1355, train/total_loss: 0.1395, train/util_ratio: 0.8125, train/run_time: 0.1422, lr: 0.0004, train/prefecth_time: 0.0043 \n",
      "1850 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.2151, train/total_loss: 0.2155, train/util_ratio: 0.6875, train/run_time: 0.1405, lr: 0.0004, train/prefecth_time: 0.0032 \n",
      "1900 iteration USE_EMA: True, train/sup_loss: 0.0412, train/unsup_loss: 0.0414, train/total_loss: 0.0826, train/util_ratio: 0.6562, train/run_time: 0.1419, lr: 0.0004, train/prefecth_time: 0.0061 \n",
      "1950 iteration USE_EMA: True, train/sup_loss: 0.0033, train/unsup_loss: 0.2867, train/total_loss: 0.2899, train/util_ratio: 0.7188, train/run_time: 0.1408, lr: 0.0004, train/prefecth_time: 0.0026 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.966 0.002 0.001 0.002 0.    0.001 0.002 0.001 0.008 0.017]\n",
      " [0.    0.965 0.    0.    0.    0.    0.    0.    0.    0.035]\n",
      " [0.055 0.    0.707 0.026 0.044 0.067 0.016 0.079 0.    0.006]\n",
      " [0.005 0.001 0.006 0.759 0.005 0.163 0.021 0.037 0.002 0.001]\n",
      " [0.006 0.    0.003 0.018 0.874 0.01  0.004 0.084 0.001 0.   ]\n",
      " [0.    0.    0.001 0.077 0.006 0.856 0.001 0.059 0.    0.   ]\n",
      " [0.004 0.    0.003 0.001 0.    0.002 0.987 0.003 0.    0.   ]\n",
      " [0.006 0.    0.001 0.009 0.07  0.101 0.    0.81  0.    0.003]\n",
      " [0.076 0.014 0.001 0.004 0.    0.    0.    0.001 0.836 0.068]\n",
      " [0.007 0.015 0.    0.    0.    0.    0.    0.    0.002 0.976]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "2000 iteration, USE_EMA: True, train/sup_loss: 0.0084, train/unsup_loss: 0.6485, train/total_loss: 0.6569, train/util_ratio: 0.7812, train/run_time: 0.1408, eval/loss: 0.3931, eval/top-1-acc: 0.8736, eval/balanced_acc: 0.8736, eval/precision: 0.8818, eval/recall: 0.8736, eval/F1: 0.8734, lr: 0.0004, train/prefecth_time: 0.0031 BEST_EVAL_ACC: 0.8736, at 2000 iters\n",
      "2050 iteration USE_EMA: True, train/sup_loss: 0.0020, train/unsup_loss: 0.1562, train/total_loss: 0.1582, train/util_ratio: 0.7500, train/run_time: 0.1408, lr: 0.0004, train/prefecth_time: 0.0034 \n",
      "2100 iteration USE_EMA: True, train/sup_loss: 0.0022, train/unsup_loss: 0.3097, train/total_loss: 0.3120, train/util_ratio: 0.8438, train/run_time: 0.1409, lr: 0.0004, train/prefecth_time: 0.0032 \n",
      "2150 iteration USE_EMA: True, train/sup_loss: 0.0062, train/unsup_loss: 0.1458, train/total_loss: 0.1520, train/util_ratio: 0.7188, train/run_time: 0.1410, lr: 0.0004, train/prefecth_time: 0.0033 \n",
      "2200 iteration USE_EMA: True, train/sup_loss: 0.0148, train/unsup_loss: 0.2113, train/total_loss: 0.2261, train/util_ratio: 0.6875, train/run_time: 0.1407, lr: 0.0004, train/prefecth_time: 0.0034 \n",
      "2250 iteration USE_EMA: True, train/sup_loss: 0.0108, train/unsup_loss: 0.2698, train/total_loss: 0.2806, train/util_ratio: 0.6250, train/run_time: 0.1403, lr: 0.0004, train/prefecth_time: 0.0031 \n",
      "2300 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.1321, train/total_loss: 0.1326, train/util_ratio: 0.8125, train/run_time: 0.1406, lr: 0.0004, train/prefecth_time: 0.0032 \n",
      "2350 iteration USE_EMA: True, train/sup_loss: 0.0099, train/unsup_loss: 0.3188, train/total_loss: 0.3287, train/util_ratio: 0.7812, train/run_time: 0.1407, lr: 0.0004, train/prefecth_time: 0.0032 \n",
      "2400 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.0553, train/total_loss: 0.0560, train/util_ratio: 0.7188, train/run_time: 0.1409, lr: 0.0004, train/prefecth_time: 0.0033 \n",
      "2450 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.3062, train/total_loss: 0.3067, train/util_ratio: 0.8125, train/run_time: 0.1406, lr: 0.0004, train/prefecth_time: 0.0032 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.968 0.002 0.001 0.001 0.    0.    0.002 0.001 0.013 0.012]\n",
      " [0.    0.979 0.    0.    0.    0.    0.    0.    0.    0.021]\n",
      " [0.061 0.    0.72  0.028 0.084 0.043 0.019 0.042 0.    0.003]\n",
      " [0.005 0.001 0.003 0.805 0.01  0.125 0.025 0.023 0.002 0.001]\n",
      " [0.004 0.    0.002 0.009 0.926 0.002 0.004 0.052 0.001 0.   ]\n",
      " [0.    0.    0.001 0.077 0.01  0.866 0.001 0.045 0.    0.   ]\n",
      " [0.004 0.    0.002 0.    0.    0.002 0.991 0.001 0.    0.   ]\n",
      " [0.008 0.    0.001 0.007 0.124 0.077 0.    0.782 0.    0.001]\n",
      " [0.036 0.008 0.    0.    0.    0.    0.    0.    0.931 0.025]\n",
      " [0.007 0.018 0.    0.    0.    0.    0.    0.    0.002 0.973]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "2500 iteration, USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.0543, train/total_loss: 0.0547, train/util_ratio: 0.5938, train/run_time: 0.1402, eval/loss: 0.3548, eval/top-1-acc: 0.8941, eval/balanced_acc: 0.8941, eval/precision: 0.8989, eval/recall: 0.8941, eval/F1: 0.8932, lr: 0.0004, train/prefecth_time: 0.0033 BEST_EVAL_ACC: 0.8941, at 2500 iters\n",
      "2550 iteration USE_EMA: True, train/sup_loss: 0.0035, train/unsup_loss: 0.1280, train/total_loss: 0.1314, train/util_ratio: 0.7812, train/run_time: 0.1404, lr: 0.0004, train/prefecth_time: 0.0031 \n",
      "2600 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.2929, train/total_loss: 0.2936, train/util_ratio: 0.8125, train/run_time: 0.1402, lr: 0.0004, train/prefecth_time: 0.0032 \n",
      "2650 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.0830, train/total_loss: 0.0832, train/util_ratio: 0.6562, train/run_time: 0.1401, lr: 0.0004, train/prefecth_time: 0.0034 \n",
      "2700 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.0310, train/total_loss: 0.0315, train/util_ratio: 0.7812, train/run_time: 0.1405, lr: 0.0004, train/prefecth_time: 0.0031 \n",
      "2750 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.2103, train/total_loss: 0.2107, train/util_ratio: 0.6562, train/run_time: 0.1396, lr: 0.0004, train/prefecth_time: 0.0034 \n",
      "2800 iteration USE_EMA: True, train/sup_loss: 0.0028, train/unsup_loss: 0.1549, train/total_loss: 0.1577, train/util_ratio: 0.7812, train/run_time: 0.1401, lr: 0.0004, train/prefecth_time: 0.0032 \n",
      "2850 iteration USE_EMA: True, train/sup_loss: 0.0170, train/unsup_loss: 0.0704, train/total_loss: 0.0874, train/util_ratio: 0.7500, train/run_time: 0.1424, lr: 0.0004, train/prefecth_time: 0.0032 \n",
      "2900 iteration USE_EMA: True, train/sup_loss: 0.0367, train/unsup_loss: 0.0295, train/total_loss: 0.0661, train/util_ratio: 0.7188, train/run_time: 0.1401, lr: 0.0003, train/prefecth_time: 0.0029 \n",
      "2950 iteration USE_EMA: True, train/sup_loss: 0.0009, train/unsup_loss: 0.2047, train/total_loss: 0.2056, train/util_ratio: 0.8438, train/run_time: 0.1405, lr: 0.0003, train/prefecth_time: 0.0031 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.968 0.002 0.001 0.001 0.    0.    0.001 0.001 0.016 0.01 ]\n",
      " [0.    0.984 0.    0.    0.    0.    0.    0.    0.    0.016]\n",
      " [0.065 0.    0.722 0.022 0.105 0.036 0.02  0.026 0.    0.004]\n",
      " [0.004 0.001 0.003 0.829 0.009 0.106 0.024 0.02  0.003 0.001]\n",
      " [0.003 0.    0.    0.005 0.946 0.001 0.005 0.039 0.001 0.   ]\n",
      " [0.    0.    0.001 0.08  0.01  0.87  0.002 0.037 0.    0.   ]\n",
      " [0.004 0.    0.002 0.    0.    0.002 0.992 0.    0.    0.   ]\n",
      " [0.009 0.    0.001 0.006 0.138 0.064 0.    0.781 0.    0.001]\n",
      " [0.023 0.008 0.    0.    0.    0.    0.    0.    0.953 0.016]\n",
      " [0.008 0.022 0.    0.    0.    0.    0.    0.    0.002 0.968]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "3000 iteration, USE_EMA: True, train/sup_loss: 0.0026, train/unsup_loss: 0.1566, train/total_loss: 0.1592, train/util_ratio: 0.7812, train/run_time: 0.1406, eval/loss: 0.3531, eval/top-1-acc: 0.9013, eval/balanced_acc: 0.9013, eval/precision: 0.9062, eval/recall: 0.9013, eval/F1: 0.9003, lr: 0.0003, train/prefecth_time: 0.0031 BEST_EVAL_ACC: 0.9013, at 3000 iters\n",
      "3050 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.3025, train/total_loss: 0.3029, train/util_ratio: 0.7500, train/run_time: 0.1398, lr: 0.0003, train/prefecth_time: 0.0033 \n",
      "3100 iteration USE_EMA: True, train/sup_loss: 0.0382, train/unsup_loss: 0.1407, train/total_loss: 0.1789, train/util_ratio: 0.6562, train/run_time: 0.1406, lr: 0.0003, train/prefecth_time: 0.0033 \n",
      "3150 iteration USE_EMA: True, train/sup_loss: 0.0020, train/unsup_loss: 0.0475, train/total_loss: 0.0495, train/util_ratio: 0.6875, train/run_time: 0.1407, lr: 0.0003, train/prefecth_time: 0.0033 \n",
      "3200 iteration USE_EMA: True, train/sup_loss: 0.0009, train/unsup_loss: 0.1359, train/total_loss: 0.1368, train/util_ratio: 0.6875, train/run_time: 0.1404, lr: 0.0003, train/prefecth_time: 0.0035 \n",
      "3250 iteration USE_EMA: True, train/sup_loss: 0.0066, train/unsup_loss: 0.1037, train/total_loss: 0.1103, train/util_ratio: 0.7812, train/run_time: 0.1404, lr: 0.0003, train/prefecth_time: 0.0033 \n",
      "3300 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.0957, train/total_loss: 0.0963, train/util_ratio: 0.6562, train/run_time: 0.1397, lr: 0.0003, train/prefecth_time: 0.0031 \n",
      "3350 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.0205, train/total_loss: 0.0210, train/util_ratio: 0.6562, train/run_time: 0.1402, lr: 0.0003, train/prefecth_time: 0.0033 \n",
      "3400 iteration USE_EMA: True, train/sup_loss: 0.1098, train/unsup_loss: 0.0330, train/total_loss: 0.1428, train/util_ratio: 0.6875, train/run_time: 0.1400, lr: 0.0003, train/prefecth_time: 0.0059 \n",
      "3450 iteration USE_EMA: True, train/sup_loss: 0.0017, train/unsup_loss: 0.1417, train/total_loss: 0.1435, train/util_ratio: 0.6875, train/run_time: 0.1410, lr: 0.0003, train/prefecth_time: 0.0031 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.968 0.002 0.001 0.001 0.    0.    0.001 0.001 0.017 0.009]\n",
      " [0.    0.986 0.    0.    0.    0.    0.    0.    0.    0.014]\n",
      " [0.063 0.    0.728 0.02  0.111 0.035 0.02  0.021 0.    0.002]\n",
      " [0.004 0.001 0.002 0.853 0.007 0.094 0.021 0.015 0.002 0.001]\n",
      " [0.003 0.    0.    0.004 0.946 0.001 0.006 0.039 0.001 0.   ]\n",
      " [0.    0.    0.001 0.067 0.009 0.888 0.002 0.033 0.    0.   ]\n",
      " [0.004 0.    0.001 0.    0.    0.002 0.993 0.    0.    0.   ]\n",
      " [0.008 0.    0.002 0.007 0.132 0.064 0.001 0.785 0.    0.001]\n",
      " [0.021 0.007 0.    0.    0.    0.    0.    0.    0.959 0.013]\n",
      " [0.006 0.023 0.    0.    0.    0.    0.    0.    0.002 0.969]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "3500 iteration, USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.0635, train/total_loss: 0.0642, train/util_ratio: 0.7188, train/run_time: 0.1406, eval/loss: 0.3464, eval/top-1-acc: 0.9075, eval/balanced_acc: 0.9075, eval/precision: 0.9124, eval/recall: 0.9075, eval/F1: 0.9066, lr: 0.0003, train/prefecth_time: 0.0033 BEST_EVAL_ACC: 0.9075, at 3500 iters\n",
      "3550 iteration USE_EMA: True, train/sup_loss: 0.0207, train/unsup_loss: 0.1370, train/total_loss: 0.1577, train/util_ratio: 0.7188, train/run_time: 0.1400, lr: 0.0003, train/prefecth_time: 0.0034 \n",
      "3600 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.2376, train/total_loss: 0.2379, train/util_ratio: 0.8125, train/run_time: 0.1406, lr: 0.0003, train/prefecth_time: 0.0034 \n",
      "3650 iteration USE_EMA: True, train/sup_loss: 0.0203, train/unsup_loss: 0.0581, train/total_loss: 0.0784, train/util_ratio: 0.6875, train/run_time: 0.1405, lr: 0.0003, train/prefecth_time: 0.0034 \n",
      "3700 iteration USE_EMA: True, train/sup_loss: 0.0291, train/unsup_loss: 0.1788, train/total_loss: 0.2078, train/util_ratio: 0.8438, train/run_time: 0.1406, lr: 0.0003, train/prefecth_time: 0.0032 \n",
      "3750 iteration USE_EMA: True, train/sup_loss: 0.0020, train/unsup_loss: 0.0720, train/total_loss: 0.0739, train/util_ratio: 0.7500, train/run_time: 0.1400, lr: 0.0003, train/prefecth_time: 0.0034 \n",
      "3800 iteration USE_EMA: True, train/sup_loss: 0.0145, train/unsup_loss: 0.1374, train/total_loss: 0.1519, train/util_ratio: 0.8750, train/run_time: 0.1407, lr: 0.0003, train/prefecth_time: 0.0033 \n",
      "3850 iteration USE_EMA: True, train/sup_loss: 0.0262, train/unsup_loss: 0.0992, train/total_loss: 0.1254, train/util_ratio: 0.7188, train/run_time: 0.1410, lr: 0.0002, train/prefecth_time: 0.0030 \n",
      "3900 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.3316, train/total_loss: 0.3319, train/util_ratio: 0.7812, train/run_time: 0.1407, lr: 0.0002, train/prefecth_time: 0.0033 \n",
      "3950 iteration USE_EMA: True, train/sup_loss: 0.0112, train/unsup_loss: 0.1994, train/total_loss: 0.2106, train/util_ratio: 0.7188, train/run_time: 0.1408, lr: 0.0002, train/prefecth_time: 0.0031 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.968 0.001 0.001 0.001 0.    0.    0.001 0.001 0.018 0.009]\n",
      " [0.    0.988 0.    0.    0.    0.    0.    0.    0.    0.012]\n",
      " [0.057 0.    0.749 0.015 0.118 0.023 0.021 0.015 0.    0.002]\n",
      " [0.005 0.001 0.001 0.87  0.008 0.077 0.025 0.01  0.002 0.001]\n",
      " [0.001 0.    0.    0.005 0.955 0.    0.008 0.03  0.001 0.   ]\n",
      " [0.    0.    0.001 0.068 0.01  0.887 0.003 0.031 0.    0.   ]\n",
      " [0.003 0.    0.001 0.001 0.    0.001 0.993 0.    0.001 0.   ]\n",
      " [0.008 0.    0.002 0.009 0.132 0.058 0.001 0.789 0.    0.001]\n",
      " [0.017 0.007 0.    0.    0.    0.    0.    0.    0.967 0.009]\n",
      " [0.006 0.025 0.    0.    0.    0.    0.    0.    0.002 0.967]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "4000 iteration, USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.5238, train/total_loss: 0.5243, train/util_ratio: 0.9062, train/run_time: 0.1405, eval/loss: 0.3374, eval/top-1-acc: 0.9133, eval/balanced_acc: 0.9133, eval/precision: 0.9181, eval/recall: 0.9133, eval/F1: 0.9125, lr: 0.0002, train/prefecth_time: 0.0032 BEST_EVAL_ACC: 0.9133, at 4000 iters\n",
      "4050 iteration USE_EMA: True, train/sup_loss: 0.0208, train/unsup_loss: 0.3388, train/total_loss: 0.3596, train/util_ratio: 0.6250, train/run_time: 0.1396, lr: 0.0002, train/prefecth_time: 0.0031 \n",
      "4100 iteration USE_EMA: True, train/sup_loss: 0.3715, train/unsup_loss: 0.0317, train/total_loss: 0.4031, train/util_ratio: 0.8438, train/run_time: 0.1403, lr: 0.0002, train/prefecth_time: 0.0034 \n",
      "4150 iteration USE_EMA: True, train/sup_loss: 0.0386, train/unsup_loss: 0.3697, train/total_loss: 0.4083, train/util_ratio: 0.6875, train/run_time: 0.1405, lr: 0.0002, train/prefecth_time: 0.0035 \n",
      "4200 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.1447, train/total_loss: 0.1449, train/util_ratio: 0.8750, train/run_time: 0.1401, lr: 0.0002, train/prefecth_time: 0.0032 \n",
      "4250 iteration USE_EMA: True, train/sup_loss: 0.0015, train/unsup_loss: 0.2502, train/total_loss: 0.2517, train/util_ratio: 0.8438, train/run_time: 0.1406, lr: 0.0002, train/prefecth_time: 0.0031 \n",
      "4300 iteration USE_EMA: True, train/sup_loss: 0.0016, train/unsup_loss: 0.4713, train/total_loss: 0.4729, train/util_ratio: 0.6562, train/run_time: 0.1403, lr: 0.0002, train/prefecth_time: 0.0033 \n",
      "4350 iteration USE_EMA: True, train/sup_loss: 0.0020, train/unsup_loss: 0.2388, train/total_loss: 0.2408, train/util_ratio: 0.7500, train/run_time: 0.1401, lr: 0.0002, train/prefecth_time: 0.0031 \n",
      "4400 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.1646, train/total_loss: 0.1653, train/util_ratio: 0.9062, train/run_time: 0.1406, lr: 0.0002, train/prefecth_time: 0.0031 \n",
      "4450 iteration USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.0542, train/total_loss: 0.0549, train/util_ratio: 0.8438, train/run_time: 0.1408, lr: 0.0002, train/prefecth_time: 0.0031 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.967 0.001 0.001 0.001 0.    0.    0.001 0.001 0.019 0.009]\n",
      " [0.    0.988 0.    0.    0.    0.    0.    0.    0.    0.012]\n",
      " [0.052 0.    0.761 0.013 0.119 0.021 0.018 0.014 0.    0.002]\n",
      " [0.005 0.001 0.001 0.881 0.008 0.068 0.025 0.008 0.002 0.001]\n",
      " [0.    0.    0.001 0.005 0.962 0.    0.008 0.023 0.001 0.   ]\n",
      " [0.    0.    0.001 0.068 0.01  0.886 0.004 0.031 0.    0.   ]\n",
      " [0.003 0.    0.001 0.002 0.    0.001 0.992 0.    0.001 0.   ]\n",
      " [0.007 0.    0.002 0.01  0.127 0.057 0.    0.795 0.    0.002]\n",
      " [0.014 0.007 0.    0.    0.    0.    0.    0.    0.97  0.009]\n",
      " [0.005 0.025 0.    0.    0.    0.    0.    0.    0.002 0.968]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "4500 iteration, USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.2390, train/total_loss: 0.2393, train/util_ratio: 0.7188, train/run_time: 0.1398, eval/loss: 0.3292, eval/top-1-acc: 0.9170, eval/balanced_acc: 0.9170, eval/precision: 0.9216, eval/recall: 0.9170, eval/F1: 0.9163, lr: 0.0002, train/prefecth_time: 0.0034 BEST_EVAL_ACC: 0.9170, at 4500 iters\n",
      "4550 iteration USE_EMA: True, train/sup_loss: 0.0047, train/unsup_loss: 0.1983, train/total_loss: 0.2030, train/util_ratio: 0.7812, train/run_time: 0.1402, lr: 0.0002, train/prefecth_time: 0.0034 \n",
      "4600 iteration USE_EMA: True, train/sup_loss: 0.1438, train/unsup_loss: 0.0164, train/total_loss: 0.1602, train/util_ratio: 0.6250, train/run_time: 0.1408, lr: 0.0002, train/prefecth_time: 0.0034 \n",
      "4650 iteration USE_EMA: True, train/sup_loss: 0.0061, train/unsup_loss: 0.1227, train/total_loss: 0.1288, train/util_ratio: 0.7188, train/run_time: 0.1407, lr: 0.0001, train/prefecth_time: 0.0033 \n",
      "4700 iteration USE_EMA: True, train/sup_loss: 0.0008, train/unsup_loss: 0.2121, train/total_loss: 0.2130, train/util_ratio: 0.7188, train/run_time: 0.1403, lr: 0.0001, train/prefecth_time: 0.0033 \n",
      "4750 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.3475, train/total_loss: 0.3480, train/util_ratio: 0.8125, train/run_time: 0.1406, lr: 0.0001, train/prefecth_time: 0.0031 \n",
      "4800 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.1207, train/total_loss: 0.1208, train/util_ratio: 0.7812, train/run_time: 0.1405, lr: 0.0001, train/prefecth_time: 0.0034 \n",
      "4850 iteration USE_EMA: True, train/sup_loss: 0.0026, train/unsup_loss: 0.1368, train/total_loss: 0.1394, train/util_ratio: 0.7812, train/run_time: 0.1402, lr: 0.0001, train/prefecth_time: 0.0032 \n",
      "4900 iteration USE_EMA: True, train/sup_loss: 0.0019, train/unsup_loss: 0.0960, train/total_loss: 0.0979, train/util_ratio: 0.5938, train/run_time: 0.1401, lr: 0.0001, train/prefecth_time: 0.0031 \n",
      "4950 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.0922, train/total_loss: 0.0929, train/util_ratio: 0.7500, train/run_time: 0.1403, lr: 0.0001, train/prefecth_time: 0.0032 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.968 0.001 0.    0.001 0.    0.    0.001 0.001 0.019 0.009]\n",
      " [0.    0.988 0.    0.    0.    0.    0.    0.    0.    0.012]\n",
      " [0.05  0.    0.774 0.008 0.119 0.02  0.018 0.01  0.    0.001]\n",
      " [0.004 0.001 0.003 0.874 0.011 0.07  0.026 0.008 0.002 0.001]\n",
      " [0.    0.    0.001 0.003 0.965 0.001 0.008 0.021 0.001 0.   ]\n",
      " [0.    0.    0.001 0.061 0.01  0.895 0.004 0.029 0.    0.   ]\n",
      " [0.003 0.    0.001 0.001 0.001 0.001 0.992 0.    0.001 0.   ]\n",
      " [0.007 0.    0.002 0.009 0.126 0.058 0.    0.796 0.    0.002]\n",
      " [0.014 0.007 0.    0.    0.    0.    0.    0.    0.972 0.007]\n",
      " [0.005 0.023 0.    0.    0.    0.    0.    0.    0.002 0.97 ]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "5000 iteration, USE_EMA: True, train/sup_loss: 0.0012, train/unsup_loss: 0.1903, train/total_loss: 0.1915, train/util_ratio: 0.7188, train/run_time: 0.1399, eval/loss: 0.3270, eval/top-1-acc: 0.9194, eval/balanced_acc: 0.9194, eval/precision: 0.9241, eval/recall: 0.9194, eval/F1: 0.9188, lr: 0.0001, train/prefecth_time: 0.0028 BEST_EVAL_ACC: 0.9194, at 5000 iters\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2023-07-19 23:45:48,260 INFO] confusion matrix\n",
      "[2023-07-19 23:45:48,261 INFO] [[0.969 0.001 0.    0.001 0.    0.    0.001 0.001 0.018 0.009]\n",
      " [0.    0.987 0.    0.    0.    0.    0.    0.    0.    0.013]\n",
      " [0.049 0.    0.776 0.007 0.127 0.017 0.016 0.007 0.    0.001]\n",
      " [0.003 0.001 0.006 0.869 0.013 0.072 0.027 0.006 0.002 0.001]\n",
      " [0.    0.    0.    0.002 0.974 0.001 0.004 0.018 0.001 0.   ]\n",
      " [0.    0.    0.001 0.045 0.011 0.916 0.003 0.024 0.    0.   ]\n",
      " [0.004 0.    0.001 0.001 0.001 0.001 0.991 0.    0.001 0.   ]\n",
      " [0.006 0.    0.002 0.006 0.127 0.059 0.    0.798 0.    0.002]\n",
      " [0.014 0.007 0.    0.    0.    0.    0.    0.    0.973 0.006]\n",
      " [0.005 0.018 0.    0.    0.    0.    0.    0.    0.002 0.975]]\n",
      "[2023-07-19 23:45:48,264 INFO] evaluation metric\n",
      "[2023-07-19 23:45:48,264 INFO] acc: 0.9228\n",
      "[2023-07-19 23:45:48,265 INFO] precision: 0.9283\n",
      "[2023-07-19 23:45:48,266 INFO] recall: 0.9228\n",
      "[2023-07-19 23:45:48,266 INFO] f1: 0.9223\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model saved: ./saved_models/fixmatch/latest_model.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2023-07-19 23:45:50,377 INFO] Best acc 0.9228 at epoch 0\n",
      "[2023-07-19 23:45:50,378 INFO] Training finished.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model saved: ./saved_models/fixmatch/model_best.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2023-07-19 23:46:52,956 INFO] confusion matrix\n",
      "[2023-07-19 23:46:52,958 INFO] [[0.969 0.001 0.    0.001 0.    0.    0.001 0.001 0.018 0.009]\n",
      " [0.    0.987 0.    0.    0.    0.    0.    0.    0.    0.013]\n",
      " [0.049 0.    0.776 0.007 0.127 0.017 0.016 0.007 0.    0.001]\n",
      " [0.003 0.001 0.006 0.869 0.013 0.072 0.027 0.006 0.002 0.001]\n",
      " [0.    0.    0.    0.002 0.974 0.001 0.004 0.018 0.001 0.   ]\n",
      " [0.    0.    0.001 0.045 0.011 0.916 0.003 0.024 0.    0.   ]\n",
      " [0.004 0.    0.001 0.001 0.001 0.001 0.991 0.    0.001 0.   ]\n",
      " [0.006 0.    0.002 0.006 0.127 0.059 0.    0.798 0.    0.002]\n",
      " [0.014 0.007 0.    0.    0.    0.    0.    0.    0.973 0.006]\n",
      " [0.005 0.018 0.    0.    0.    0.    0.    0.    0.002 0.975]]\n",
      "[2023-07-19 23:46:52,961 INFO] evaluation metric\n",
      "[2023-07-19 23:46:52,961 INFO] acc: 0.9228\n",
      "[2023-07-19 23:46:52,962 INFO] precision: 0.9283\n",
      "[2023-07-19 23:46:52,963 INFO] recall: 0.9228\n",
      "[2023-07-19 23:46:52,963 INFO] f1: 0.9223\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'acc': 0.9228,\n",
       " 'precision': 0.928316578833672,\n",
       " 'recall': 0.9228,\n",
       " 'f1': 0.9223419792678316}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training and evaluation\n",
    "trainer = Trainer(config, algorithm)\n",
    "trainer.fit(train_lb_loader, train_ulb_loader, eval_loader)\n",
    "trainer.evaluate(eval_loader)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('test')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "vscode": {
   "interpreter": {
    "hash": "efd87a861e5021e4a438e5b61d692cea261dd91508182bfdfdb13fb969975ffe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
